package com.xy.annotaion.handler.kotlin.spider

import com.xy.annotaion.handler.kotlin.p
import org.jsoup.Jsoup
import java.io.BufferedOutputStream
import java.io.FileOutputStream

fun main() {
    val userAgent =
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4638.69 Safari/537.36"

    //100MB cache http request data
    val maxBytes = 1024 * 1024 * 100
    val file = BufferedOutputStream(FileOutputStream("2021近视防控科学大会上午场.mp4"))

    var homeUrl = "pull-sh1.weizan.cn"
    val videoPhaseList =
        "https://pull-sh1.weizan.cn/1335967082/405756996311079426/replay.1636689571.82604732.m3u8"
    //jsoup 下载文件默认1MB数据，需要设置才能满足要求
    val videoListOriginData =
        Jsoup.connect(videoPhaseList)
            .header("user-agent", userAgent)
            .ignoreContentType(true)
            .maxBodySize(maxBytes)
            .execute()
            .body()

    p(videoListOriginData)
    var counter = 0;
    val originDataList = videoListOriginData.split("\n")
    for (i in originDataList) {
        p(i);
        if (i?.isNotBlank() && !i.startsWith("#")) {

            val videoPhaseData =
                Jsoup.connect(i)
                    .header("user-agent", userAgent)
                    .ignoreContentType(true)
                    .maxBodySize(maxBytes)
                    .execute()
                    .bodyAsBytes()
            p("data size:", videoPhaseData.size)
            file.write(videoPhaseData)
            counter++
            if (counter % 20 == 0) {
                file.flush()
            }
        }
    }
    file.flush()
    file.close()

}
